package us.codecraft.webmagic.samples;

import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import ipproxy.IPModel.IPMessage;
import ipproxy.database.MyRedis;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.selector.Html;

public class dianyingProcessor implements PageProcessor {
	int i = 0;
	static MyRedis redis = new MyRedis();

	@Override
	public void process(Page page) {
		System.out.println(i++);
		// 防止url跟着友情链接之类的外网链接跑偏了
		List<String> urls = page.getHtml().links().regex(".*80s.*").all();
		// 如果是打开列表页的链接则新开页面
		// TODO 规则还没写 http://.+\\/\\w+/\\d+
		if (page.getUrl().regex("https?://.+\\/\\w+/\\d+").match()) {
			List<String> moviesDetails = page.getHtml().xpath("//div[@id='body']/div").all();
			for (String movieDetail : moviesDetails) {
				if (new Html(movieDetail).xpath("//div[@class='info']/h1[@class='font14w']/tidyText()") != null) {
					page.putField("title",
							new Html(movieDetail).xpath("//div[@class='info']/h1[@class='font14w']/tidyText()"));
					page.putField("content", new Html(movieDetail).xpath("//div[@id='movie_content']/tidyText()"));
					page.putField("downUrl", new Html(movieDetail)
							.xpath("//ul[@class='dllist1']//li[@class='clearfix dlurlelement']//a/@href"));
				}

			}
		} else {
			page.addTargetRequests(urls);
		}

	}

	@Override
	public Site getSite() {
		return Site.me().setDomain("80s.tw").setCharset("utf-8")
				.setUserAgent(
						"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
				.setSleepTime(800).setRetryTimes(3).setTimeOut(3000);
	}

	public static void main(String[] args) {
		List<IPMessage> list = redis.getIPByList("IPPool");
		List<Proxy> proxies = new ArrayList<>();
		for (IPMessage ip : list) {
			Proxy proxy = new Proxy(ip.getIPAddress(), Integer.parseInt(ip.getIPPort()));
			proxies.add(proxy);
		}
		//TODO 设置代理ip之后返回500
		SimpleProxyProvider provider = new SimpleProxyProvider(proxies);
		HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
		httpClientDownloader.setProxyProvider(provider);
		Spider.create(new dianyingProcessor()).addUrl("http://www.80s.tw/")
				.addPipeline(new ConsolePipeline()).thread(10).run();
	}
}
